import requests
import re
import sqlite3
from multiprocessing import Process,Pool

def write_sql(title,content,img):
    conn = sqlite3.connect('db.sqlite3')
    cursor = conn.cursor()
    try:
        cursor.execute('insert into neihanduanzi ("title","content","img") values ("{}","{}","{}")'.format(title,content,img))
        conn.commit()
    except:
        pass
    cursor.close()
def get_links(max,processes):
    for num in range(2,max):
        page_url='http://www.neihanpa.com/article/index_{}.html'.format(num)
        print("爬取第{}页...".format(num))
        response = requests.get(page_url)
        response.encoding = 'utf-8'
        pagetext=response.text
        pageresult=re.findall(r'<div class="text-column-item box box-790">([\s\S]*?)</div>',pagetext)
        links=[]
        for i in pageresult:
            link=re.findall(r'<h3><a href="(.*?)"',i)[0]
            links.append(link)
        pool = Pool(processes=processes)
        for link in links:
            pool.apply_async(download_one, (link,))
        pool.close()
        pool.join()
def download_one(link):
    page_url = 'http://www.neihanpa.com{}'.format(link)
    response = requests.get(page_url)
    response.encoding = 'utf-8'
    pagetext = response.text
    title=re.findall(r'<h1 class="title">(.*?)</h1>',pagetext)[0]
    print(title)
    detail=re.findall(r'<div class="detail">([\s\S]*)<div class="ad610">',pagetext)[0]
    img=re.findall(r'<img src="(.*?)"',detail)[0]
    content=re.findall(r'</a></p>([\s\S]*)',detail)[0]
    write_sql(title,content,img)

if __name__ == "__main__":
    max=int(input('下载多少页？'))
    processes=4
    get_links(max,processes)